{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Brainstorming and Focus Group Quantitative Experimentation 2.1:**Difficult people** under **action correction** + **divergence intervention**\n",
    "\n",
    "Can we use TinyTroupe to brainstorm product ideas?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "!!!!\n",
      "DISCLAIMER: TinyTroupe relies on Artificial Intelligence (AI) models to generate content. \n",
      "The AI models are not perfect and may produce inappropriate or inacurate results. \n",
      "For any serious or consequential use, please review the generated content before using it.\n",
      "!!!!\n",
      "\n",
      "Looking for default config on: c:\\Users\\pdasilva\\AppData\\Local\\anaconda3\\envs\\py310\\lib\\site-packages\\tinytroupe\\utils\\..\\config.ini\n",
      "Found custom config on: c:\\Users\\pdasilva\\OneDrive - Microsoft\\TinyTroupe (shared)\\Paper artifacts\\Working examples\\config.ini\n",
      "\n",
      "=================================\n",
      "Current TinyTroupe configuration \n",
      "=================================\n",
      "[OpenAI]\n",
      "api_type = openai\n",
      "azure_api_version = 2024-08-01-preview\n",
      "model = gpt-4o-mini\n",
      "reasoning_model = o3-mini\n",
      "embedding_model = text-embedding-3-small\n",
      "max_tokens = 16000\n",
      "temperature = 1.2\n",
      "freq_penalty = 0.0\n",
      "presence_penalty = 0.0\n",
      "timeout = 60\n",
      "max_attempts = 5\n",
      "waiting_time = 0\n",
      "exponential_backoff_factor = 5\n",
      "reasoning_effort = high\n",
      "cache_api_calls = False\n",
      "cache_file_name = openai_api_cache.pickle\n",
      "max_content_display_length = 1024\n",
      "azure_embedding_model_api_version = 2023-05-15\n",
      "\n",
      "[Simulation]\n",
      "parallel_agent_generation = True\n",
      "parallel_agent_actions = True\n",
      "rai_harmful_content_prevention = True\n",
      "rai_copyright_infringement_prevention = True\n",
      "\n",
      "[ActionGenerator]\n",
      "max_attempts = 2\n",
      "enable_quality_checks = False\n",
      "enable_regeneration = True\n",
      "enable_direct_correction = False\n",
      "enable_quality_check_for_persona_adherence = True\n",
      "enable_quality_check_for_selfconsistency = False\n",
      "enable_quality_check_for_fluency = False\n",
      "enable_quality_check_for_suitability = False\n",
      "enable_quality_check_for_similarity = False\n",
      "continue_on_failure = True\n",
      "quality_threshold = 5\n",
      "\n",
      "[Logging]\n",
      "loglevel = WARNING\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "from tinytroupe.agent import TinyPerson\n",
    "from tinytroupe.environment import TinyWorld\n",
    "from tinytroupe.experimentation import InPlaceExperimentRunner\n",
    "from tinytroupe.steering import Intervention\n",
    "from tinytroupe.examples import *\n",
    "from tinytroupe.validation import propositions\n",
    "from tinytroupe.extraction import ResultsExtractor\n",
    "from tinytroupe.utils.parallel import parallel_map_dict, parallel_map_cross\n",
    "from tinytroupe.validation import hard_persona_adherence, persona_adherence, self_consistency, fluency, task_completion, divergence\n",
    "\n",
    "# specific utilities\n",
    "from common_utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_mode = False  # set to True to run the full mode with all agents and tasks\n",
    "\n",
    "# avoid displaying the communication, to make the output cleaner for eval\n",
    "TinyPerson.communication_display = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if full_mode:\n",
    "    repetitions_per_task = 5\n",
    "    simulation_steps = 15\n",
    "\n",
    "else:\n",
    "    repetitions_per_task = 2\n",
    "    simulation_steps = 10\n",
    "    qty_agents = 12\n",
    "    qty_proposals = 4\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-06-03 21:19:23,406 - tinytroupe - WARNING - Configuration file './brainstorming_and_focus_group_quantitative_experimentation_2.1c.json' exists and was loaded successfully. If you are trying to fully rerun the experiments, delete it first.\n"
     ]
    }
   ],
   "source": [
    "experiment_runner = InPlaceExperimentRunner(\"./brainstorming_and_focus_group_quantitative_experimentation_2.1c.json\")\n",
    "\n",
    "experiment_runner.add_experiment(\"Control\")\n",
    "experiment_runner.add_experiment(\"Treatment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_runner.activate_next_experiment()\n",
    "\n",
    "#experiment_runner.fix_active_experiment(\"Control\")\n",
    "#experiment_runner.fix_active_experiment(\"Treatment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running experiment None\n"
     ]
    }
   ],
   "source": [
    "print(f\"Running experiment {experiment_runner.get_active_experiment()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agents and populations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "people = []\n",
    "if not experiment_runner.has_finished_all_experiments():\n",
    "    # load agents\n",
    "    people = TinyPerson.load_specifications_from_folder(\"./population/difficult_people\")\n",
    "\n",
    "    # filter to make it go faster?\n",
    "    if not full_mode:\n",
    "        people = people[:qty_agents]\n",
    "\n",
    "    # customize and print minibios \n",
    "    for person in people:\n",
    "        person.import_fragment(\"./fragments/difficult_person.agent.fragment.json\")\n",
    "        print(person.minibio(extended=False))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(people)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# divide people in several groups of 5\n",
    "people_groups = []\n",
    "for i in range(0, len(people), 4):\n",
    "    people_groups.append(people[i:i+4]\n",
    "    )\n",
    "\n",
    "len(people_groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The experiment refers to customers\n",
    "\n",
    "if experiment_runner.get_active_experiment() == \"Control\":\n",
    "    for person in people:\n",
    "        person.action_generator.enable_reasoning_step = False\n",
    "        person.action_generator.enable_quality_checks = False\n",
    "\n",
    "elif experiment_runner.get_active_experiment() == \"Treatment\":    \n",
    "    for person in people:\n",
    "       person.action_generator.enable_reasoning_step = False\n",
    "       person.action_generator.enable_quality_checks = True\n",
    "       person.action_generator.max_attempts = 2\n",
    "       person.action_generator.enable_regeneration = True\n",
    "       person.action_generator.quality_threshold = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Proposals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "proposals = [\n",
    "    {\"theme\": \"Daily Life and Convenience\",\n",
    "     \"objective\": \"Create ideas for products or services that simplify, enhance, or bring joy to everyday tasks, routines, and interactions.\"},\n",
    "\n",
    "    {\"theme\": \"Personal Growth and Wellbeing\",\n",
    "     \"objective\": \"Generate concepts for products or experiences that support personal development, health, mental wellness, emotional care, or community connection.\"},\n",
    "\n",
    "    {\"theme\": \"Discovery and Exploration\",\n",
    "     \"objective\": \"Explore ideas for products, services, or platforms that encourage curiosity, learning, adventure, and exploration of both the external world and inner self.\"},\n",
    "\n",
    "    {\"theme\": \"Productivity and Resourcefulness\",\n",
    "     \"objective\": \"Invent new tools, processes, or organizational systems that empower people or groups to achieve more, optimize resources, or collaborate effectively.\"},\n",
    "\n",
    "    {\"theme\": \"Creativity and Expression\",\n",
    "     \"objective\": \"Design ideas for new products, platforms, or services that inspire creativity, foster expression, enhance artistic skills, or enable new forms of storytelling and communication.\"}\n",
    "]\n",
    "\n",
    "if not full_mode:\n",
    "    proposals = proposals[:qty_proposals]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[{'theme': 'Daily Life and Convenience',\n",
       "   'objective': 'Create ideas for products or services that simplify, enhance, or bring joy to everyday tasks, routines, and interactions.'},\n",
       "  {'theme': 'Personal Growth and Wellbeing',\n",
       "   'objective': 'Generate concepts for products or experiences that support personal development, health, mental wellness, emotional care, or community connection.'}],\n",
       " [{'theme': 'Discovery and Exploration',\n",
       "   'objective': 'Explore ideas for products, services, or platforms that encourage curiosity, learning, adventure, and exploration of both the external world and inner self.'},\n",
       "  {'theme': 'Productivity and Resourcefulness',\n",
       "   'objective': 'Invent new tools, processes, or organizational systems that empower people or groups to achieve more, optimize resources, or collaborate effectively.'}]]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# divide the proposals in exactly two groups (half/half)\n",
    "proposals_groups = []\n",
    "proposals_groups.append(proposals[:len(proposals)//2])\n",
    "proposals_groups.append(proposals[len(proposals)//2:])\n",
    "\n",
    "proposals_groups"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Auxiliary functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def brainstorming_battery(agents, proposals, interventions, agent_propositions, environment_propositions, \n",
    "                          repetitions = 5, simulation_steps=10): \n",
    "    \n",
    "    agent_propositions_scores = {}\n",
    "    environment_propositions_scores = {}\n",
    "\n",
    "    experiments_count = 0\n",
    "    total_expected_experiments = len(proposals) * repetitions #* len(agents)\n",
    "\n",
    "    # loop over proposals and repetitions\n",
    "    for proposal in proposals:\n",
    "\n",
    "        objective = proposal[\"objective\"]\n",
    "        theme = proposal[\"theme\"]\n",
    "\n",
    "        for i in range(repetitions):\n",
    "            print(\"\\n############## STARTING A NEW RESEARCH SESSION #################\")\n",
    "            print(f\"Overall experiment number: {experiments_count+1} / {total_expected_experiments}\")\n",
    "            print(f\"Discussion objective: {objective}\")\n",
    "            print(f\"Trial number: {i+1}\")\n",
    "            print(f\"Agents: {agents}\")\n",
    "\n",
    "            # clear the episodic memory of all agents\n",
    "            for person in agents:\n",
    "                person.clear_episodic_memory()\n",
    "\n",
    "            world = TinyWorld(agents=agents, interventions=interventions)\n",
    "            \n",
    "            # Participants introduce themselves\n",
    "            world.broadcast(f\"\"\"\n",
    "                Hello everyone! Let's start by introducing ourselves, and mentioning problems we face in our daily personal\n",
    "                and professional lives related to the following theme: {theme}\n",
    "                \n",
    "                Please:\n",
    "                  - present yourself and your background;\n",
    "                  - present some key personal problems related to the theme;\n",
    "                  - present some key problems related to the theme that you face in your work;\n",
    "                  - present some key problems related to the theme that you see in your industry as a whole.\n",
    "                  \n",
    "                Don't discuss solutions yet, just the problems you face and see others facing.\n",
    "                \"\"\")\n",
    "            world.run(1)\n",
    "            \n",
    "            # now to the brainstorming session itself\n",
    "            world.broadcast(f\"\"\"\n",
    "                Folks, your mission is to brainstorm {objective}. \n",
    "                Please follow these guidelines:\n",
    "                  - give a unique and informative name to each idea you propose, so that it is easy to refer to it. Say it like \"Idea name: '<name of the idea>'\".;\n",
    "                  - explain why you think it is a good idea, and what problem it solves, and how you feel about it;\n",
    "                  - your ideas should be new complete, self-contained, products or services, not features for other existing products or services;\n",
    "                  - think of creative ideas that would somehow help you in both in your personal and professional lives.\n",
    "                  - create as many different and unique ideas as you can during the brainstorming session. Each idea must be **completely** different from the others \n",
    "                    (either by yourself or by others), and not just a variation of an existing idea.\n",
    "                    and not just a variation of an existing idea.\n",
    "                  - you should criticize each other's ideas, in order to make sure they are as\n",
    "                    good as possible, but no more than once per idea.\n",
    "                  - you should also provide suggestions for improvement to each other's ideas, in order to make them as good as possible, \n",
    "                    but no more than once per idea.\n",
    "                  - regardless of critique or complement, you **must** primarily propose new ideas quickly, \n",
    "                    not just build on existing ones. \n",
    "                  - propose one idea at a time, instead of proposing multiple ideas at once, to allow appropriate discussion.\n",
    "                  - you should **not** propose ideas that are too similar to each other, or to the ones already proposed by others.\n",
    "                  - before saying anything, THINK deeply about yourself, your beliefs, interests, needs, life, etc., to come up with ideas that are\n",
    "                    truly unique and different from the ones already proposed by others.\n",
    "                   \n",
    "                Please start the discussion now.\n",
    "                \"\"\")\n",
    "            world.run(simulation_steps)\n",
    "\n",
    "            # extract and count ideas\n",
    "            rapporteur = agents[0]  # the first agent is the rapporteur\n",
    "            rapporteur.listen_and_act(\"Can you please consolidate the ideas that the group came up with? Provide a lot of details on each idea, and complement anything missing.\")\n",
    "            ideas = ResultsExtractor().extract_results_from_agent(rapporteur, \n",
    "                                    extraction_objective=\"Consolidates the ideas that the group came up with, explaining each idea as an item of a list.\" \\\n",
    "                                                        \"Add information about: what problem the idea solves; to which target audience it is meant.\" \\\n",
    "                                                        \"how is it different from competing, existing, products.\", \n",
    "                                    situation=\"A focus group to brainstorm new product ideas.\",\n",
    "                                    fields= [\"name\", \"description\", \"problem\", \"target_audience\", \"competition_analysis\"],\n",
    "                                    fields_hints={\"ideas\": \"must be the root of the resulting dictionary.\"},)\n",
    "            pprint(ideas)\n",
    "            if \"ideas_qty\" not in environment_propositions_scores:\n",
    "                environment_propositions_scores[\"ideas_qty\"] = []\n",
    "            if ideas is not None and \"ideas\" in ideas and isinstance(ideas[\"ideas\"], list):\n",
    "                environment_propositions_scores[\"ideas_qty\"].append(len(ideas[\"ideas\"]))\n",
    "\n",
    "            # Evaluate environment propositions in parallel\n",
    "            env_results = parallel_map_dict(\n",
    "                environment_propositions,\n",
    "                lambda item: item[1].copy().score(\n",
    "                    world, \n",
    "                    claim_variables={\"task_description\": f\"A brainstorming or focus group session was run about: {objective}.\"}, \n",
    "                    return_full_response=True\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            # Process environment results\n",
    "            for k, result in env_results.items():\n",
    "                if k not in environment_propositions_scores:\n",
    "                    environment_propositions_scores[k] = []\n",
    "                environment_propositions_scores[k].append(result[\"value\"])\n",
    "                print(\"value: \", result[\"value\"])\n",
    "                print(\"justification: \", result[\"justification\"])\n",
    "                print(\"reasoning: \", result[\"reasoning\"])\n",
    "\n",
    "            # Evaluate agent propositions across all agents in parallel\n",
    "            agent_results = parallel_map_cross(\n",
    "                [agents, agent_propositions.items()],\n",
    "                lambda agent, prop_item: (\n",
    "                    prop_item[0],  # proposition key\n",
    "                    prop_item[1].copy().score(agent, return_full_response=True)  # result\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            # Process agent results\n",
    "            for k, result in agent_results:\n",
    "                if k not in agent_propositions_scores:\n",
    "                    agent_propositions_scores[k] = []\n",
    "                if result is not None:\n",
    "                    agent_propositions_scores[k].append(result[\"value\"])\n",
    "                    print(\"value: \", result[\"value\"])\n",
    "                    print(\"justification: \", result[\"justification\"])\n",
    "                    print(\"reasoning: \", result[\"reasoning\"])\n",
    "                    print(\"\\n\\n\")\n",
    "                else:\n",
    "                    print(f\"*****WARNING:***** Agent did not respond to proposition {k}.\")\n",
    "            #\n",
    "            ##for k, proposition in agent_propositions.items():\n",
    "            ##    for person in world.agents:\n",
    "            ##        result = proposition.copy().score(person, return_full_response=True)\n",
    "            ##        \n",
    "            ##        if k not in agent_propositions_scores:\n",
    "            ##            agent_propositions_scores[k] = []\n",
    "            ##        agent_propositions_scores[k].append(result[\"value\"])\n",
    "            ##\n",
    "            ##        print(\"value: \", result[\"value\"])\n",
    "            ##        print(\"justification: \", result[\"justification\"])\n",
    "            ##        print(\"reasoning: \", result[\"reasoning\"])\n",
    "            ##        print(\"\\n\\n\")\n",
    "            ##\n",
    "            \n",
    "            experiments_count += 1\n",
    "            print(\"\\n\\n\")\n",
    "\n",
    "    return agent_propositions_scores, environment_propositions_scores\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent_propositions_scores={}\n",
    "environment_propositions_scores={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def brainstorm(people, proposals=proposals):\n",
    "    global agent_propositions_scores, environment_propositions_scores\n",
    "    if not experiment_runner.has_finished_all_experiments():\n",
    "\n",
    "        interventions = []\n",
    "        if experiment_runner.get_active_experiment() == \"Treatment\":\n",
    "            interventions = \\\n",
    "                Intervention.create_for_each(people)\\\n",
    "                    .set_functional_precondition(lambda target: target.actions_count >=7)\\\n",
    "                    .set_textual_precondition(\n",
    "                        \"\"\"\n",
    "                        AGENT IS NOT PROPOSING COMPLETELY NEW PRODUCT/SERVICE IDEAS ANYMORE:\n",
    "                        The last **entirely** new product/service idea proposed by this agent, if any, was proposed by him/her **more** than 5 of simulation events ago.\n",
    "                        That is to say, the agent has not proposed any new product/service idea in the last 5 of his/her simulation trajectory events.\n",
    "                        Additional features, variations of or other refinements to product/service ideas already proposed are NOT considered new!\n",
    "\n",
    "                        How to compute the steps gap:\n",
    "                        1. Determine the current next event number (N); and the last event number in which the agent proposed a new product/service idea (M).\n",
    "                            This information can be found in the simulation trajectory.\n",
    "                        2. Compute the **difference** beteween the current next event number and the last event number in which the agent proposed a new product/service idea: D = N - M\n",
    "                        3. The proposition is true if, and only if, the difference D is **greater than** 5.\n",
    "                        \"\"\")\\\n",
    "                    .set_effect(lambda target: target.think(\"\"\"\n",
    "                                                            I need to propose additional, **completelly** new and different, product/service ideas. This was part of the requirement for this session.\n",
    "                                                            I will propose an entirely **new** idea now, I **cannot** repeat or refine previous ideas! I cannot make variations\n",
    "                                                            of previous ideas (e.g., \"XYZ for A\", \"XYZ for B\", \"XYZ for Z\" are repetitive, there should be only one \"XYZ\"), \n",
    "                                                            I need to think of something **entirely** new and different.\n",
    "                                                            To help me avoid repeating previous ideas, I'll now explicitly THINK about all the ideas already given by myself or\n",
    "                                                            others, and then, based on that, I'll think again about a new unique idea.\n",
    "                                                            \"\"\"))\n",
    "\n",
    "                                                            \n",
    "        tmp_agent_propositions_scores, tmp_environment_propositions_scores = \\\n",
    "            brainstorming_battery(\n",
    "                agents=people,\n",
    "                proposals=proposals,\n",
    "                interventions=interventions,    \n",
    "                agent_propositions={\n",
    "                    \"Hard Persona Adherence\": hard_persona_adherence,\n",
    "                    \"Self-consistency\": self_consistency,\n",
    "                    \"Fluency\": fluency\n",
    "                },\n",
    "                environment_propositions={\n",
    "                    \"Task Completion\": task_completion,\n",
    "                    \"Divergence\": divergence\n",
    "                },\n",
    "                repetitions=repetitions_per_task,\n",
    "                simulation_steps=simulation_steps\n",
    "            )\n",
    "\n",
    "        pprint(\"NEW AGENT PROPOSITIONS SCORES\")\n",
    "        pprint(tmp_agent_propositions_scores)\n",
    "        print(\"\\n\\n\")\n",
    "        pprint(\"NEW ENVIRONMENT PROPOSITIONS SCORES\")\n",
    "        pprint(tmp_environment_propositions_scores)\n",
    "\n",
    "        # merge the scores lists\n",
    "        agent_propositions_scores = merge_dicts_of_lists(tmp_agent_propositions_scores, agent_propositions_scores)\n",
    "        environment_propositions_scores = merge_dicts_of_lists(tmp_environment_propositions_scores, environment_propositions_scores)\n",
    "\n",
    "        return agent_propositions_scores, environment_propositions_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[0], proposals_groups[0]) if len(people_groups) > 0  and len(proposals_groups) > 0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[0], proposals_groups[1]) if len(people_groups) > 0  and len(proposals_groups) > 1 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[1], proposals_groups[0]) if len(people_groups) > 1  and len(proposals_groups) > 0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[1], proposals_groups[1]) if len(people_groups) > 1  and len(proposals_groups) > 1 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[2], proposals_groups[0]) if len(people_groups) > 2  and len(proposals_groups) > 0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[2], proposals_groups[1]) if len(people_groups) > 2  and len(proposals_groups) > 1 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[3], proposals_groups[0]) if len(people_groups) > 3  and len(proposals_groups) > 0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[3], proposals_groups[1]) if len(people_groups) > 3  and len(proposals_groups) > 1 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[4], proposals_groups[0]) if len(people_groups) > 4  and len(proposals_groups) > 0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[4], proposals_groups[1]) if len(people_groups) > 4  and len(proposals_groups) > 1 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract results and analyze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment finished. No more experiments to run.\n"
     ]
    }
   ],
   "source": [
    "if experiment_runner.get_active_experiment() in [\"Control\", \"Treatment\"]:\n",
    "    combined_scores = {**agent_propositions_scores, **environment_propositions_scores}\n",
    "    experiment_runner.add_experiment_results(combined_scores, experiment_name=experiment_runner.get_active_experiment()) \n",
    "    \n",
    "    plot_scores(combined_scores)\n",
    "\n",
    "else:\n",
    "    print(\"Experiment finished. No more experiments to run.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All experiments have been finished.\n",
      "STATISTICTS: Control vs\n",
      "{'Treatment': {'Divergence': {'confidence_interval': (-2.3439782869763954,\n",
      "                                                      -0.4893550463569387),\n",
      "                              'confidence_level': 0.95,\n",
      "                              'control_mean': 6.291666666666667,\n",
      "                              'control_sample_size': 24,\n",
      "                              'control_std': 1.1220775245117838,\n",
      "                              'degrees_of_freedom': 36.83183157810619,\n",
      "                              'effect_size': -0.8937148693650034,\n",
      "                              'mean_difference': -1.416666666666667,\n",
      "                              'p_value': 0.003739146546083642,\n",
      "                              'percent_change': -22.51655629139073,\n",
      "                              'significant': True,\n",
      "                              't_statistic': 3.095919122439936,\n",
      "                              'test_type': 'Welch t-test (unequal variance)',\n",
      "                              'treatment_mean': 4.875,\n",
      "                              'treatment_sample_size': 24,\n",
      "                              'treatment_std': 1.9406968716999795},\n",
      "               'Fluency': {'confidence_interval': (-3.523424526868247,\n",
      "                                                   -2.726575473131753),\n",
      "                           'confidence_level': 0.95,\n",
      "                           'control_mean': 6.34375,\n",
      "                           'control_sample_size': 96,\n",
      "                           'control_std': 1.3520209045409726,\n",
      "                           'degrees_of_freedom': 189.16294369789034,\n",
      "                           'effect_size': -2.2331550223706347,\n",
      "                           'mean_difference': -3.125,\n",
      "                           'p_value': 1.9801669888818148e-35,\n",
      "                           'percent_change': -49.26108374384236,\n",
      "                           'significant': True,\n",
      "                           't_statistic': 15.471751839694209,\n",
      "                           'test_type': 'Welch t-test (unequal variance)',\n",
      "                           'treatment_mean': 3.21875,\n",
      "                           'treatment_sample_size': 96,\n",
      "                           'treatment_std': 1.4451597981210462},\n",
      "               'Hard Persona Adherence': {'confidence_interval': (0.4461706509028963,\n",
      "                                                                  1.1579960157637714),\n",
      "                                          'confidence_level': 0.95,\n",
      "                                          'control_mean': 4.145833333333333,\n",
      "                                          'control_sample_size': 96,\n",
      "                                          'control_std': 1.3915471636893697,\n",
      "                                          'degrees_of_freedom': 179.6489505214888,\n",
      "                                          'effect_size': 0.641858126593057,\n",
      "                                          'mean_difference': 0.8020833333333339,\n",
      "                                          'p_value': 1.5227315306856247e-05,\n",
      "                                          'percent_change': 19.346733668341724,\n",
      "                                          'significant': True,\n",
      "                                          't_statistic': -4.446923546040604,\n",
      "                                          'test_type': 'Welch t-test (unequal '\n",
      "                                                       'variance)',\n",
      "                                          'treatment_mean': 4.947916666666667,\n",
      "                                          'treatment_sample_size': 96,\n",
      "                                          'treatment_std': 1.0893725056840524},\n",
      "               'Self-consistency': {'confidence_interval': (-2.482160939606376,\n",
      "                                                            -1.6428390603936247),\n",
      "                                    'confidence_level': 0.95,\n",
      "                                    'control_mean': 5.604166666666667,\n",
      "                                    'control_sample_size': 96,\n",
      "                                    'control_std': 1.5112153816794487,\n",
      "                                    'degrees_of_freedom': 189.50369428810922,\n",
      "                                    'effect_size': -1.3992821596192555,\n",
      "                                    'mean_difference': -2.0625000000000004,\n",
      "                                    'p_value': 2.6560792331228735e-18,\n",
      "                                    'percent_change': -36.802973977695174,\n",
      "                                    'significant': True,\n",
      "                                    't_statistic': 9.694511178341017,\n",
      "                                    'test_type': 'Welch t-test (unequal '\n",
      "                                                 'variance)',\n",
      "                                    'treatment_mean': 3.5416666666666665,\n",
      "                                    'treatment_sample_size': 96,\n",
      "                                    'treatment_std': 1.4357588616379597},\n",
      "               'Task Completion': {'confidence_interval': (-4.665173552980815,\n",
      "                                                           -2.251493113685851),\n",
      "                                   'confidence_level': 0.95,\n",
      "                                   'control_mean': 5.958333333333333,\n",
      "                                   'control_sample_size': 24,\n",
      "                                   'control_std': 2.422344654087937,\n",
      "                                   'degrees_of_freedom': 40.44970073430188,\n",
      "                                   'effect_size': -1.6713160591940852,\n",
      "                                   'mean_difference': -3.458333333333333,\n",
      "                                   'p_value': 9.003416294139811e-07,\n",
      "                                   'percent_change': -58.04195804195804,\n",
      "                                   'significant': True,\n",
      "                                   't_statistic': 5.789608660059899,\n",
      "                                   'test_type': 'Welch t-test (unequal '\n",
      "                                                'variance)',\n",
      "                                   'treatment_mean': 2.5,\n",
      "                                   'treatment_sample_size': 24,\n",
      "                                   'treatment_std': 1.6418441381303657},\n",
      "               'ideas_qty': {'confidence_interval': (-2.960227689778024,\n",
      "                                                     -0.4731056435553094),\n",
      "                             'confidence_level': 0.95,\n",
      "                             'control_mean': 4.166666666666667,\n",
      "                             'control_sample_size': 24,\n",
      "                             'control_std': 2.4963741822833803,\n",
      "                             'degrees_of_freedom': 38.94488524915237,\n",
      "                             'effect_size': -0.8107830863217459,\n",
      "                             'mean_difference': -1.7166666666666668,\n",
      "                             'p_value': 0.008069265615129632,\n",
      "                             'percent_change': -41.199999999999996,\n",
      "                             'significant': True,\n",
      "                             't_statistic': 2.7923383898223704,\n",
      "                             'test_type': 'Welch t-test (unequal variance)',\n",
      "                             'treatment_mean': 2.45,\n",
      "                             'treatment_sample_size': 20,\n",
      "                             'treatment_std': 1.538112308540638}}}\n",
      "{'Divergence': [6,\n",
      "                7,\n",
      "                6,\n",
      "                7,\n",
      "                6,\n",
      "                6,\n",
      "                7,\n",
      "                6,\n",
      "                7,\n",
      "                6,\n",
      "                7,\n",
      "                7,\n",
      "                8,\n",
      "                8,\n",
      "                6,\n",
      "                7,\n",
      "                7,\n",
      "                4,\n",
      "                6,\n",
      "                5,\n",
      "                7,\n",
      "                6,\n",
      "                3,\n",
      "                6],\n",
      " 'Fluency': [6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             6,\n",
      "             9,\n",
      "             7,\n",
      "             8,\n",
      "             6,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             4,\n",
      "             4,\n",
      "             7,\n",
      "             5,\n",
      "             2,\n",
      "             1,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             9,\n",
      "             6,\n",
      "             6,\n",
      "             9,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             5,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             5,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             3,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             5,\n",
      "             3,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             4,\n",
      "             5,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             8,\n",
      "             4,\n",
      "             6,\n",
      "             6,\n",
      "             6,\n",
      "             9,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             8],\n",
      " 'Hard Persona Adherence': [4,\n",
      "                            6,\n",
      "                            4,\n",
      "                            3,\n",
      "                            4,\n",
      "                            4,\n",
      "                            5,\n",
      "                            3,\n",
      "                            4,\n",
      "                            3,\n",
      "                            5,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            4,\n",
      "                            4,\n",
      "                            4,\n",
      "                            4,\n",
      "                            4,\n",
      "                            3,\n",
      "                            5,\n",
      "                            2,\n",
      "                            2,\n",
      "                            5,\n",
      "                            4,\n",
      "                            3,\n",
      "                            3,\n",
      "                            4,\n",
      "                            5,\n",
      "                            5,\n",
      "                            5,\n",
      "                            3,\n",
      "                            2,\n",
      "                            4,\n",
      "                            2,\n",
      "                            4,\n",
      "                            5,\n",
      "                            4,\n",
      "                            4,\n",
      "                            7,\n",
      "                            5,\n",
      "                            2,\n",
      "                            3,\n",
      "                            5,\n",
      "                            4,\n",
      "                            5,\n",
      "                            2,\n",
      "                            4,\n",
      "                            5,\n",
      "                            2,\n",
      "                            5,\n",
      "                            6,\n",
      "                            1,\n",
      "                            3,\n",
      "                            3,\n",
      "                            5,\n",
      "                            5,\n",
      "                            2,\n",
      "                            2,\n",
      "                            3,\n",
      "                            4,\n",
      "                            5,\n",
      "                            4,\n",
      "                            3,\n",
      "                            7,\n",
      "                            2,\n",
      "                            2,\n",
      "                            4,\n",
      "                            4,\n",
      "                            6,\n",
      "                            3,\n",
      "                            5,\n",
      "                            2,\n",
      "                            7,\n",
      "                            3,\n",
      "                            6,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            4,\n",
      "                            5,\n",
      "                            5,\n",
      "                            4,\n",
      "                            4,\n",
      "                            5,\n",
      "                            4,\n",
      "                            3,\n",
      "                            5,\n",
      "                            3,\n",
      "                            6,\n",
      "                            3,\n",
      "                            3,\n",
      "                            5,\n",
      "                            6],\n",
      " 'Self-consistency': [4,\n",
      "                      3,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      5,\n",
      "                      6,\n",
      "                      7,\n",
      "                      4,\n",
      "                      8,\n",
      "                      2,\n",
      "                      3,\n",
      "                      6,\n",
      "                      5,\n",
      "                      3,\n",
      "                      5,\n",
      "                      7,\n",
      "                      7,\n",
      "                      4,\n",
      "                      3,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      5,\n",
      "                      6,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      8,\n",
      "                      5,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      4,\n",
      "                      7,\n",
      "                      7,\n",
      "                      5,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      3,\n",
      "                      5,\n",
      "                      7,\n",
      "                      5,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      5,\n",
      "                      6,\n",
      "                      6,\n",
      "                      3,\n",
      "                      9,\n",
      "                      4,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      2,\n",
      "                      5,\n",
      "                      5,\n",
      "                      2,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      4,\n",
      "                      4,\n",
      "                      2,\n",
      "                      5,\n",
      "                      7,\n",
      "                      4,\n",
      "                      5,\n",
      "                      6,\n",
      "                      4,\n",
      "                      3,\n",
      "                      7,\n",
      "                      6,\n",
      "                      5,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      3],\n",
      " 'Task Completion': [7,\n",
      "                     7,\n",
      "                     6,\n",
      "                     1,\n",
      "                     7,\n",
      "                     2,\n",
      "                     4,\n",
      "                     2,\n",
      "                     7,\n",
      "                     7,\n",
      "                     8,\n",
      "                     7,\n",
      "                     7,\n",
      "                     9,\n",
      "                     6,\n",
      "                     8,\n",
      "                     3,\n",
      "                     7,\n",
      "                     6,\n",
      "                     8,\n",
      "                     1,\n",
      "                     7,\n",
      "                     9,\n",
      "                     7],\n",
      " 'ideas_qty': [5,\n",
      "               4,\n",
      "               4,\n",
      "               3,\n",
      "               3,\n",
      "               3,\n",
      "               1,\n",
      "               4,\n",
      "               3,\n",
      "               2,\n",
      "               3,\n",
      "               2,\n",
      "               3,\n",
      "               3,\n",
      "               5,\n",
      "               12,\n",
      "               3,\n",
      "               8,\n",
      "               3,\n",
      "               9,\n",
      "               7,\n",
      "               4,\n",
      "               3,\n",
      "               3]}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Proposition</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>Standard Deviation</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hard Persona Adherence</td>\n",
       "      <td>4.145833</td>\n",
       "      <td>1.391547</td>\n",
       "      <td>96.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Self-consistency</td>\n",
       "      <td>5.604167</td>\n",
       "      <td>1.511215</td>\n",
       "      <td>96.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Fluency</td>\n",
       "      <td>6.343750</td>\n",
       "      <td>1.352021</td>\n",
       "      <td>96.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ideas_qty</td>\n",
       "      <td>4.166667</td>\n",
       "      <td>2.496374</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Task Completion</td>\n",
       "      <td>5.958333</td>\n",
       "      <td>2.422345</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Divergence</td>\n",
       "      <td>6.291667</td>\n",
       "      <td>1.122078</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Proposition  Average Score  Standard Deviation  Count\n",
       "0  Hard Persona Adherence       4.145833            1.391547   96.0\n",
       "1        Self-consistency       5.604167            1.511215   96.0\n",
       "2                 Fluency       6.343750            1.352021   96.0\n",
       "3               ideas_qty       4.166667            2.496374   24.0\n",
       "4         Task Completion       5.958333            2.422345   24.0\n",
       "5              Divergence       6.291667            1.122078   24.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Divergence': [6,\n",
      "                6,\n",
      "                7,\n",
      "                7,\n",
      "                4,\n",
      "                2,\n",
      "                2,\n",
      "                5,\n",
      "                1,\n",
      "                6,\n",
      "                6,\n",
      "                1,\n",
      "                4,\n",
      "                3,\n",
      "                7,\n",
      "                4,\n",
      "                4,\n",
      "                4,\n",
      "                6,\n",
      "                7,\n",
      "                6,\n",
      "                6,\n",
      "                7,\n",
      "                6],\n",
      " 'Fluency': [4,\n",
      "             3,\n",
      "             4,\n",
      "             4,\n",
      "             8,\n",
      "             3,\n",
      "             5,\n",
      "             2,\n",
      "             3,\n",
      "             5,\n",
      "             7,\n",
      "             4,\n",
      "             4,\n",
      "             8,\n",
      "             4,\n",
      "             2,\n",
      "             3,\n",
      "             3,\n",
      "             4,\n",
      "             2,\n",
      "             2,\n",
      "             2,\n",
      "             1,\n",
      "             2,\n",
      "             2,\n",
      "             4,\n",
      "             5,\n",
      "             2,\n",
      "             1,\n",
      "             4,\n",
      "             3,\n",
      "             3,\n",
      "             2,\n",
      "             3,\n",
      "             6,\n",
      "             2,\n",
      "             5,\n",
      "             3,\n",
      "             3,\n",
      "             3,\n",
      "             4,\n",
      "             4,\n",
      "             4,\n",
      "             2,\n",
      "             2,\n",
      "             4,\n",
      "             3,\n",
      "             3,\n",
      "             2,\n",
      "             1,\n",
      "             5,\n",
      "             3,\n",
      "             3,\n",
      "             4,\n",
      "             2,\n",
      "             2,\n",
      "             2,\n",
      "             2,\n",
      "             4,\n",
      "             2,\n",
      "             5,\n",
      "             3,\n",
      "             4,\n",
      "             4,\n",
      "             1,\n",
      "             2,\n",
      "             4,\n",
      "             2,\n",
      "             5,\n",
      "             2,\n",
      "             2,\n",
      "             2,\n",
      "             5,\n",
      "             2,\n",
      "             4,\n",
      "             5,\n",
      "             2,\n",
      "             2,\n",
      "             2,\n",
      "             3,\n",
      "             5,\n",
      "             2,\n",
      "             1,\n",
      "             5,\n",
      "             4,\n",
      "             3,\n",
      "             6,\n",
      "             2,\n",
      "             3,\n",
      "             2,\n",
      "             4,\n",
      "             2,\n",
      "             4,\n",
      "             3,\n",
      "             2,\n",
      "             2],\n",
      " 'Hard Persona Adherence': [4,\n",
      "                            5,\n",
      "                            4,\n",
      "                            4,\n",
      "                            5,\n",
      "                            6,\n",
      "                            7,\n",
      "                            4,\n",
      "                            6,\n",
      "                            3,\n",
      "                            6,\n",
      "                            5,\n",
      "                            5,\n",
      "                            5,\n",
      "                            4,\n",
      "                            4,\n",
      "                            5,\n",
      "                            5,\n",
      "                            5,\n",
      "                            4,\n",
      "                            5,\n",
      "                            4,\n",
      "                            5,\n",
      "                            4,\n",
      "                            5,\n",
      "                            7,\n",
      "                            4,\n",
      "                            3,\n",
      "                            5,\n",
      "                            5,\n",
      "                            5,\n",
      "                            5,\n",
      "                            3,\n",
      "                            6,\n",
      "                            5,\n",
      "                            6,\n",
      "                            6,\n",
      "                            6,\n",
      "                            3,\n",
      "                            5,\n",
      "                            3,\n",
      "                            5,\n",
      "                            4,\n",
      "                            5,\n",
      "                            6,\n",
      "                            5,\n",
      "                            5,\n",
      "                            5,\n",
      "                            4,\n",
      "                            4,\n",
      "                            7,\n",
      "                            8,\n",
      "                            5,\n",
      "                            6,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            3,\n",
      "                            4,\n",
      "                            5,\n",
      "                            4,\n",
      "                            4,\n",
      "                            3,\n",
      "                            4,\n",
      "                            4,\n",
      "                            4,\n",
      "                            5,\n",
      "                            7,\n",
      "                            4,\n",
      "                            4,\n",
      "                            5,\n",
      "                            5,\n",
      "                            6,\n",
      "                            4,\n",
      "                            7,\n",
      "                            5,\n",
      "                            5,\n",
      "                            5,\n",
      "                            4,\n",
      "                            5,\n",
      "                            7,\n",
      "                            5,\n",
      "                            5,\n",
      "                            6,\n",
      "                            5,\n",
      "                            5,\n",
      "                            4,\n",
      "                            6,\n",
      "                            6,\n",
      "                            6,\n",
      "                            5,\n",
      "                            6,\n",
      "                            5,\n",
      "                            3],\n",
      " 'Self-consistency': [2,\n",
      "                      3,\n",
      "                      5,\n",
      "                      6,\n",
      "                      2,\n",
      "                      5,\n",
      "                      6,\n",
      "                      4,\n",
      "                      6,\n",
      "                      3,\n",
      "                      6,\n",
      "                      2,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      3,\n",
      "                      5,\n",
      "                      1,\n",
      "                      2,\n",
      "                      6,\n",
      "                      3,\n",
      "                      2,\n",
      "                      3,\n",
      "                      1,\n",
      "                      3,\n",
      "                      3,\n",
      "                      5,\n",
      "                      7,\n",
      "                      2,\n",
      "                      3,\n",
      "                      3,\n",
      "                      2,\n",
      "                      4,\n",
      "                      6,\n",
      "                      6,\n",
      "                      3,\n",
      "                      5,\n",
      "                      3,\n",
      "                      3,\n",
      "                      3,\n",
      "                      4,\n",
      "                      6,\n",
      "                      6,\n",
      "                      2,\n",
      "                      6,\n",
      "                      3,\n",
      "                      5,\n",
      "                      3,\n",
      "                      3,\n",
      "                      2,\n",
      "                      5,\n",
      "                      3,\n",
      "                      4,\n",
      "                      4,\n",
      "                      3,\n",
      "                      3,\n",
      "                      4,\n",
      "                      3,\n",
      "                      3,\n",
      "                      4,\n",
      "                      5,\n",
      "                      2,\n",
      "                      2,\n",
      "                      2,\n",
      "                      3,\n",
      "                      3,\n",
      "                      4,\n",
      "                      3,\n",
      "                      6,\n",
      "                      3,\n",
      "                      3,\n",
      "                      3,\n",
      "                      2,\n",
      "                      4,\n",
      "                      3,\n",
      "                      3,\n",
      "                      2,\n",
      "                      2,\n",
      "                      5,\n",
      "                      5,\n",
      "                      2,\n",
      "                      3,\n",
      "                      2,\n",
      "                      4,\n",
      "                      3,\n",
      "                      2,\n",
      "                      4,\n",
      "                      2,\n",
      "                      4,\n",
      "                      2,\n",
      "                      2,\n",
      "                      3,\n",
      "                      2,\n",
      "                      3,\n",
      "                      3,\n",
      "                      3],\n",
      " 'Task Completion': [1,\n",
      "                     0,\n",
      "                     4,\n",
      "                     4,\n",
      "                     3,\n",
      "                     1,\n",
      "                     0,\n",
      "                     4,\n",
      "                     5,\n",
      "                     4,\n",
      "                     4,\n",
      "                     3,\n",
      "                     2,\n",
      "                     1,\n",
      "                     4,\n",
      "                     4,\n",
      "                     2,\n",
      "                     2,\n",
      "                     4,\n",
      "                     1,\n",
      "                     3,\n",
      "                     0,\n",
      "                     4,\n",
      "                     0],\n",
      " 'ideas_qty': [4, 4, 2, 0, 1, 2, 4, 4, 2, 1, 1, 3, 3, 2, 3, 3, 0, 6, 3, 1]}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Proposition</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>Standard Deviation</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hard Persona Adherence</td>\n",
       "      <td>4.947917</td>\n",
       "      <td>1.089373</td>\n",
       "      <td>96.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Self-consistency</td>\n",
       "      <td>3.541667</td>\n",
       "      <td>1.435759</td>\n",
       "      <td>96.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Fluency</td>\n",
       "      <td>3.218750</td>\n",
       "      <td>1.445160</td>\n",
       "      <td>96.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ideas_qty</td>\n",
       "      <td>2.450000</td>\n",
       "      <td>1.538112</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Task Completion</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>1.641844</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Divergence</td>\n",
       "      <td>4.875000</td>\n",
       "      <td>1.940697</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Proposition  Average Score  Standard Deviation  Count\n",
       "0  Hard Persona Adherence       4.947917            1.089373   96.0\n",
       "1        Self-consistency       3.541667            1.435759   96.0\n",
       "2                 Fluency       3.218750            1.445160   96.0\n",
       "3               ideas_qty       2.450000            1.538112   20.0\n",
       "4         Task Completion       2.500000            1.641844   24.0\n",
       "5              Divergence       4.875000            1.940697   24.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if experiment_runner.has_finished_all_experiments():\n",
    "    print(\"All experiments have been finished.\")\n",
    "    print(f\"STATISTICTS: Control vs\")\n",
    "    pprint(experiment_runner.run_statistical_tests(control_experiment_name='Control'))\n",
    "\n",
    "    # plot scores of both experiments\n",
    "    experiment_control_scores = experiment_runner.get_experiment_results(\"Control\")\n",
    "    experiment_treatment_scores = experiment_runner.get_experiment_results(\"Treatment\")\n",
    "    \n",
    "    \n",
    "    plot_scores(experiment_control_scores)\n",
    "    plot_scores(experiment_treatment_scores)\n",
    "\n",
    "else:\n",
    "    print(\"Not all experiments have been finished. RESTART AND RERUN.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiment_runner.finish_active_experiment()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
